#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 20200212
# Project: qgggzy
import hashlib
import json
import math
import uuid
import pymysql
from datetime import datetime
from datetime import timedelta

from pyspider.libs.base_handler import *


class Handler(BaseHandler):
    crawl_config = {
        'proxy':'127.0.0.1:3128',
        'headers': {
            'User-Agent:':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
        }
       
    }

    def __init__(self):
        self.db = pymysql.connect(host='106.13.85.158',port=3310,user='root', passwd='root@123', db='python', charset='utf8',use_unicode=True)

    @every(minutes=60)
    def on_start(self):
        self.crawl('http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp',
                   callback=self.query_page,data=self.assemblyData('1'),method="POST")
    #分页查询
    #任务的有效期。在此期间，该页面将被视为未修改。默认值：-1（从不重新抓取）  单位秒
    @config(age=0)
    def query_page(self, response):
        if response.json['success'] == True or response.json['success'] == 'true':
            pageNumber = response.json['ttlpage']
            for index in range(int(pageNumber)):
            # 由于pyspider的url策略  url callback不同 才生成一个请求 故添加?v=index
                self.crawl(
                    'http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp?v='+str(index),
                    callback=self.index_page,method="POST",data=self.assemblyData(str(index+1)))
        
    #查询每一页的详情
    @config(age=0)
    def index_page(self, response):
        for data in response.json['data']:
            url = data['url']
            self.crawl(url.replace('/a/','/b/') , callback=self.detail_page,save=data)
    #跳转到详情页面
    @config(priority=2)
    def detail_page(self,response):
        self.saveOrUpdate(response.save,response.text)
        return response.save
    # 保存或者更新	
    def saveOrUpdate(self,data,html):
        classify = data['classify']
        classifyShow = pymysql.escape_string(data['classifyShow'])
        districtShow = pymysql.escape_string(data['districtShow'])
        platformName = pymysql.escape_string(data['platformName'])
        stageName = pymysql.escape_string(data['stageName'])
        stageShow = pymysql.escape_string(data['stageShow'])
        timeShow = data['timeShow']
        title = data['title']
        tradeShow = data['tradeShow']
        url = data['url']
        md5id = self.md5(url+title)
        selectSql = 'select * from T_RESULT_INFO t where t.source_id_md5 = "'+md5id+'"'
        currTime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        insertSql = 'insert into T_RESULT_INFO(id, source_id_md5,URL,TITLE, TYPE,UPDATE_USER,UPDATE_TIME,CREATE_USER,CREATE_TIME,UP_ID,UP_SOURCE_ID,CLASSIFY,CLASSIFY_SHOW,DISTRICT_SHOW,PLATFORM_NAME,STAGE_NAME,STAGE_SHOW,TIME_SHOW,TRADE_SHOW,HTML) ' \
                  'values ("%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")' % (
            uuid.uuid1(),md5id,url, pymysql.escape_string(title),'0',
            '','','',currTime,'','',classify,classifyShow,districtShow,platformName,stageName,stageShow,timeShow,tradeShow,pymysql.escape_string(str(html)));
        updateSql = 'update  T_RESULT_INFO set TITLE="%s", UPDATE_USER="%s",UPDATE_TIME="%s",UP_ID="%s",UP_SOURCE_ID="%s" where source_id_md5="%s"' \
            '' % (pymysql.escape_string(title),'', currTime,'', '', md5id);
        try:
            cursor = self.db.cursor()
            cursor.execute(selectSql)
            results = cursor.fetchall()
            print(results)
            if len(results) >0:
                print('update sql')
                cursor.execute(updateSql)
            else:
                print('insert sql')
                cursor.execute(insertSql)
            self.db.commit()
        except Exception as e:
            print(e)
            self.db.rollback()
    def assemblyData(self,pageNumber):
        data = {}
        # 默认取10天
        currTime = datetime.now().strftime('%Y-%m-%d')
        beginTime = (datetime.now() + timedelta(days=-9)).strftime('%Y-%m-%d')
        data['DEAL_TIME'] = '02'
        data['SOURCE_TYPE'] = '1'
        data["TIMEEND_SHOW"] = currTime
        data["TIMEBEGIN_SHOW"] = beginTime
        data["TIMEBEGIN"] = beginTime
        data["TIMEEND"] = beginTime
        data["PAGENUMBER"] = pageNumber
        data["isShowAll"] = '1'
        data["DEAL_CLASSIFY"] = '00'
        data["DEAL_STAGE"] = '0000'
        data["DEAL_PROVINCE"] = '0'
        data["DEAL_CITY"] = '0'
        data["DEAL_STAGE"] = '0000'
        data['DEAL_PLATFORM'] = '0'
        data['BID_PLATFORM'] = '0'
        data['DEAL_TRADE'] = '0'
        data['FINDTXT'] = ''
        return data
    def md5(self,str):
        m = hashlib.md5()
        m.update(str.encode("utf8"))
        return m.hexdigest()
